{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python374jvsc74a57bd02a7e95a32014fc1ccf24626d45a98c6e7b4373277259c22f47a91d487fc3e8a5",
   "display_name": "Python 3.7.4 64-bit ('base': conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "output_type": "error",
     "ename": "IndexError",
     "evalue": "list index out of range",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-93-b1c6f4225d94>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     39\u001b[0m   \u001b[0mCourseLists\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mres\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'result'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'list'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     40\u001b[0m   \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m20\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m    \u001b[1;31m# 每页实际有20个课程，提取前20两个课程的ID\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 41\u001b[1;33m     \u001b[0mcourseName\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mCourseLists\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'mocCourseBaseCardVo'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'name'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     42\u001b[0m     \u001b[0mschoolName\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mCourseLists\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'mocCourseBaseCardVo'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'schoolName'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     43\u001b[0m     \u001b[0mcourseID\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mCourseLists\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'id'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "# 实现断点续传,待检验\n",
    "import requests,json\n",
    "import time\n",
    "from lxml import etree  \n",
    "from selenium import  webdriver   \n",
    "from selenium.webdriver.chrome.options import Options\n",
    "\n",
    "chrome_options = Options() # 实例化Option对象\n",
    "chrome_options.add_argument('--headless') # 把Chrome浏览器设置为【静默模式】\n",
    "driver = webdriver.Chrome(r'C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe',options = chrome_options) # 设置引擎为Chrome，后台默默运行\n",
    "\n",
    "for coursePage in range(1,67):    #提取国家精品前5页的课程，实际为66页\n",
    "  headers = {\n",
    "      'authority': 'www.icourse163.org',\n",
    "      'sec-ch-ua': '^\\\\^',\n",
    "      'sec-ch-ua-mobile': '?0',\n",
    "      'edu-script-token': '9a13f48d9d9441c7a52eb08e5425f9de',\n",
    "      'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36',\n",
    "      'content-type': 'application/x-www-form-urlencoded;charset=UTF-8',\n",
    "      'accept': '*/*',\n",
    "      'origin': 'https://www.icourse163.org',\n",
    "      'sec-fetch-site': 'same-origin',\n",
    "      'sec-fetch-mode': 'cors',\n",
    "      'sec-fetch-dest': 'empty',\n",
    "      'referer': 'https://www.icourse163.org/channel/2001.htm',\n",
    "      'accept-language': 'zh-CN,zh;q=0.9',\n",
    "      'cookie': 'NTESSTUDYSI=9a13f48d9d9441c7a52eb08e5425f9de; EDUWEBDEVICE=6f0ee68ac9a44d449d037e10e66e265e; Hm_lvt_77dc9a9d49448cf5e629e5bebaa5500b=1620530716; __yadk_uid=dFA6XacL7KlykZc5LToZmz0r2QGOLS9X; WM_NI=HaFrWzSHdfRFUCq^%^2BCmudxZBBiN5Xbwj1SjR6yUlJ1mmLoLHyWE5sR3Ax2qV8^%^2BG5sqvkIgSxylf1NHyyhE1YylN1nusZODtSM0cHTi^%^2B8qCjr^%^2FN2FMqDek4ZUSW9c4g4u7TUc^%^3D; WM_NIKE=9ca17ae2e6ffcda170e2e6eeb1fc7982b4b885cf68adb08eb2c44e839b9aaab53cba99e597d45e93bbaeaaf52af0fea7c3b92a8cbf8a93d6729198ac87d14e979dbfccc267b7bcb8bbc14aab92fbb9e17daaef9d8bd13ebb8ba4abb467af94a1adcb73959bfb94d13983a8fe92d534e999fb94f47ea997a9b4fb73b0baffafcb73bbb88bd2dc73b092ff8ff8799a918c95d247f4bd88b5ec21fcaf8d8bf1258eee8aaef450a7b5fcbbd85995e7add5e640b7e99cb6e237e2a3; WM_TID=SiLrJ6^%^2B^%^2BqSdBRRBVBFd7xnHqqVQBKbMr; NTES_YD_SESS=X44tT.SAlKciKAMW5XcyN1a.MCrtoTzjPNvqzF_aBBLVAZ_WA7Kw6g6PpF0L_aogceKfCKPR9RN6h6o6K4YyU7jknUsLq5S4sDKC8meW8M_UMm62T9uG1aLJI4ajxn1HJybCpgLRvHdHgEzFiFsnUExhKOzc9ZNP6WgAiLdxx_wQq0Yr2IRR2eu6ro0nC.kP2FAz1eT8XNgEy.SDWrLwVjz6wf.lBLP69qgmqEcTbnk53; NTES_YD_PASSPORT=zDiybY8e3J8iOpkKO7ZIF7TfvFh1BAuInocxPS8cJC25NK67NXivTdT.A_Cu6bmdhziVFi.WwW3YYk8mo6CTlfIRKyQHgOny8ej78PcFDJQawVqU3Or.IdWmzr6UaKTCOY0zoQgzIjglm54YG9_PfGpkYhbepHrWJl0sMZ2zQp3J8opN8RiwunDttq0Iql1NeZKByMkUejvPEVKC3d7lLM7B2; S_INFO=1620544023^|0^|3&80^#^#^|18571826829; P_INFO=18571826829^|1620544023^|1^|imooc^|00&99^|US&1618926254&study^#US&null^#10^#0^#0^|&0^|null^|18571826829; STUDY_INFO=^\\\\^yd.f13d5e27bafe41f79^@163.com^|8^|1135858868^|1620544023994^\\\\^; STUDY_SESS=^\\\\^bVgiYAmYewA6l8YvVUjDaDSBesHqbbfBw9OauOtbNwypn9Jumx7P2GDBDSCKCFGz/vRlk2SsJVdKSGSHhAKVcgIp2wWoOSoe6AeXJQAUVVVflkCjbNnl6DDkHZNe4Gyx8m49RclMVBLhg397qRSTz5gKkbrGso2DR7quj9DvN9wLhur2Nm2wEb9HcEikV+3FTI8+lZKyHhiycNQo+g+/oA==^\\\\^; STUDY_PERSIST=^\\\\^XbWZdTYnK/oZnSgNQ+fHcJqNDqM3eI4Ia8MYlAywFbpmkB0TFdgXm0jlJU4KvS1iI5whnYQ7U0cPpRO8MIhGPI9g1f4SlzQ+ioi7jvJ3rpwfikQrRYWP8W0Qtueva71jxFDatn7TU/IuWqAFGSM4W0H1VGizRXqNXw4vqIFjj4cM6wopp9AonIQD2dZH2G5tof4CY3ecgBSEXLRERdTIuGr/eHDnpOOE0dWGnJTtn+HZgpjCC7Iso4RP9U87vJE8LtaQzUT1ovP2MqtW5+L3Hw+PvH8+tZRDonbf7gEH7JU=^\\\\^; NETEASE_WDA_UID=1135858868^#^|^#1519567762363; Hm_lpvt_77dc9a9d49448cf5e629e5bebaa5500b=1620544044',\n",
    "  }\n",
    "\n",
    "  params = (\n",
    "      ('csrfKey', '9a13f48d9d9441c7a52eb08e5425f9de'),\n",
    "  )\n",
    "\n",
    "  data = {\n",
    "    'mocCourseQueryVo': '{\"categoryId\":-1,\"categoryChannelId\":2001,\"orderBy\":0,\"stats\":30,\"pageIndex\":'+str(coursePage)+',\"pageSize\":20}'\n",
    "  }\n",
    "\n",
    "  res = requests.post('https://www.icourse163.org/web/j/mocSearchBean.searchCourseCardByChannelAndCategoryId.rpc', headers=headers, params=params, data=data)\n",
    "  CourseLists = res.json()['result']['list']\n",
    "  for i in range(20):    # 每页实际有20个课程，提取前20两个课程的ID\n",
    "    courseName = CourseLists[i]['mocCourseBaseCardVo']['name']\n",
    "    schoolName = CourseLists[i]['mocCourseBaseCardVo']['schoolName']\n",
    "    courseID = CourseLists[i]['id']\n",
    "    schoolID = CourseLists[i]['mocCourseBaseCardVo']['schoolSN']\n",
    "    # print(courseName,schoolName)\n",
    "    course = {}\n",
    "    course['courseName'] = courseName\n",
    "    course['schoolName'] = schoolName   # 爬取课程名称ID,学校名称ID\n",
    "\n",
    "    with open('courseID log.txt','r',encoding='utf-8') as f_log:\n",
    "      ID = f_log.read()     #FileNotFoundError\n",
    "\n",
    "    if str(courseID) not in ID:    # 判断日志里是否有内容,没有就开始爬虫\n",
    "\n",
    "      contentPage = 1   # 从评论第一页开始提取\n",
    "      # while True:\n",
    "\n",
    "      data2 = {\n",
    "        'courseId': courseID,\n",
    "      }\n",
    "\n",
    "      data3 = {\n",
    "        'courseId': courseID,\n",
    "        'pageIndex': contentPage,\n",
    "        'pageSize': '20',\n",
    "        'orderBy': '3'\n",
    "      }\n",
    "      \n",
    "      driver.get('https://www.icourse163.org/course/{}-{}'.format(schoolID,courseID)) \n",
    "      time.sleep(1)  \n",
    "      text = driver.page_source   # 渲染网页HTML,方便xpath提取\n",
    "      rxml = etree.HTML(text)\n",
    "      course['detail'] = rxml.xpath(\"//div[@id='j-course-heading-intro']/div[@id='j-rectxt2']/text()\")  # 提取课程详情\n",
    "\n",
    "      res_2 = requests.post('https://www.icourse163.org/web/j/mocCourseV2RpcBean.getEvaluateAvgAndCount.rpc', headers=headers, params=params, data=data2)\n",
    "      course['contentCount'] = res_2.json()['result']['evaluateCount']  # 提取评论数量\n",
    "\n",
    "      res_3 = requests.post('https://www.icourse163.org/web/j/mocCourseV2RpcBean.getCourseEvaluatePaginationByCourseIdOrTermId.rpc', headers=headers, params=params, data=data3)\n",
    "      contents = res_3.json()['result']['list'][:5]\n",
    "      course['content'] = []\n",
    "      for content in contents:\n",
    "        course['content'].append(content['content'])  # 提取前5个评论\n",
    "\n",
    "      # print (course) \n",
    "      data = json.dumps(course,ensure_ascii=False).encode('utf-8').decode('utf-8') \n",
    "      with open('course.json','a',encoding='utf-8') as file_obj:\n",
    "        file_obj.write(data)\n",
    "        file_obj.write('\\n')\n",
    "\n",
    "      # 断点日志\n",
    "      with open('courseID log.txt','a',encoding='utf-8') as f_log:\n",
    "        f_log.write(str(courseID))\n",
    "        f_log.write('\\n')\n",
    "\n",
    "    # # 提取所有评论数据\n",
    "    # if contents != []:  \n",
    "    #   for content in contents:\n",
    "    #       course['content'].append(content['content'])\n",
    "    #   contentPage += 1\n",
    "    # else :    #当评论为空时中断循环\n",
    "    #   contentPage = 1\n",
    "    #   # print (course)  #将课程信息存为json或者打印出来\n",
    "    #   data = json.dumps(course,ensure_ascii=False).encode('utf-8').decode('utf-8') \n",
    "    #   with open('course.json','a',encoding='utf-8') as file_obj:\n",
    "    #     file_obj.write(data)\n",
    "    #   break   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "course = {'courseName': '生物医药实验室安全知识', 'schoolName': '暨南大学', 'content': ['我说我在实验室干过很多不安全的事情你敢信？？？', '老师很仔细', '安全无小事，要重视实验室安全。']}\n",
    "# with open('course.json','a') as file_obj:\n",
    "#     json.dump(course,file_obj,ensure_ascii=False)       # 存在乱码,\n",
    "data = json.dumps(course,ensure_ascii=False).encode('utf-8').decode('utf-8')          # 不存在乱码\n",
    "print(data)\n",
    "with open('course.json','a',encoding='utf-8') as file_obj:\n",
    "    file_obj.write(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('courseID log.txt','r',encoding='utf-8') as f_log:\n",
    "    ID = f_log.read()\n",
    "if '323' in ID:\n",
    "    print('yes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}